{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import sys\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.000000    1264764\n",
       "0.166667     138501\n",
       "0.200000     107492\n",
       "0.300000      59098\n",
       "0.400000      50013\n",
       "             ...   \n",
       "0.813022          1\n",
       "0.967863          1\n",
       "0.897840          1\n",
       "0.945655          1\n",
       "0.872187          1\n",
       "Name: target, Length: 2913, dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(r'G:\\First_NLP\\jigsaw-unintended-bias-in-toxicity-classification\\train.csv',sep = None)\n",
    "\n",
    "train['target'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False,  True, False])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['target'].values > 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>severe_toxicity</th>\n",
       "      <th>obscene</th>\n",
       "      <th>identity_attack</th>\n",
       "      <th>insult</th>\n",
       "      <th>threat</th>\n",
       "      <th>asian</th>\n",
       "      <th>atheist</th>\n",
       "      <th>...</th>\n",
       "      <th>article_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>funny</th>\n",
       "      <th>wow</th>\n",
       "      <th>sad</th>\n",
       "      <th>likes</th>\n",
       "      <th>disagree</th>\n",
       "      <th>sexual_explicit</th>\n",
       "      <th>identity_annotator_count</th>\n",
       "      <th>toxicity_annotator_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>59848</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>This is so cool. It's like, 'would you want yo...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2006</td>\n",
       "      <td>rejected</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>59849</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Thank you!! This would make my life a lot less...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2006</td>\n",
       "      <td>rejected</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>59852</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>This is such an urgent design problem; kudos t...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2006</td>\n",
       "      <td>rejected</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>59855</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Is this something I'll be able to install on m...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2006</td>\n",
       "      <td>rejected</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>59856</td>\n",
       "      <td>0.893617</td>\n",
       "      <td>haha you guys are a bunch of losers.</td>\n",
       "      <td>0.021277</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021277</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2006</td>\n",
       "      <td>rejected</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1804869</th>\n",
       "      <td>6333967</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Maybe the tax on \"things\" would be collected w...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>399385</td>\n",
       "      <td>approved</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1804870</th>\n",
       "      <td>6333969</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>What do you call people who STILL think the di...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>399528</td>\n",
       "      <td>approved</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1804871</th>\n",
       "      <td>6333982</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>thank you ,,,right or wrong,,, i am following ...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>399457</td>\n",
       "      <td>approved</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1804872</th>\n",
       "      <td>6334009</td>\n",
       "      <td>0.621212</td>\n",
       "      <td>Anyone who is quoted as having the following e...</td>\n",
       "      <td>0.030303</td>\n",
       "      <td>0.030303</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.621212</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>399519</td>\n",
       "      <td>approved</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1804873</th>\n",
       "      <td>6334010</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Students defined as EBD are legally just as di...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>399318</td>\n",
       "      <td>approved</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1804874 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id    target                                       comment_text  \\\n",
       "0          59848  0.000000  This is so cool. It's like, 'would you want yo...   \n",
       "1          59849  0.000000  Thank you!! This would make my life a lot less...   \n",
       "2          59852  0.000000  This is such an urgent design problem; kudos t...   \n",
       "3          59855  0.000000  Is this something I'll be able to install on m...   \n",
       "4          59856  0.893617               haha you guys are a bunch of losers.   \n",
       "...          ...       ...                                                ...   \n",
       "1804869  6333967  0.000000  Maybe the tax on \"things\" would be collected w...   \n",
       "1804870  6333969  0.000000  What do you call people who STILL think the di...   \n",
       "1804871  6333982  0.000000  thank you ,,,right or wrong,,, i am following ...   \n",
       "1804872  6334009  0.621212  Anyone who is quoted as having the following e...   \n",
       "1804873  6334010  0.000000  Students defined as EBD are legally just as di...   \n",
       "\n",
       "         severe_toxicity   obscene  identity_attack    insult  threat  asian  \\\n",
       "0               0.000000  0.000000         0.000000  0.000000     0.0    NaN   \n",
       "1               0.000000  0.000000         0.000000  0.000000     0.0    NaN   \n",
       "2               0.000000  0.000000         0.000000  0.000000     0.0    NaN   \n",
       "3               0.000000  0.000000         0.000000  0.000000     0.0    NaN   \n",
       "4               0.021277  0.000000         0.021277  0.872340     0.0    0.0   \n",
       "...                  ...       ...              ...       ...     ...    ...   \n",
       "1804869         0.000000  0.000000         0.000000  0.000000     0.0    NaN   \n",
       "1804870         0.000000  0.000000         0.000000  0.000000     0.0    NaN   \n",
       "1804871         0.000000  0.000000         0.000000  0.000000     0.0    NaN   \n",
       "1804872         0.030303  0.030303         0.045455  0.621212     0.0    NaN   \n",
       "1804873         0.000000  0.000000         0.000000  0.000000     0.0    NaN   \n",
       "\n",
       "         atheist  ...  article_id    rating  funny  wow  sad  likes  disagree  \\\n",
       "0            NaN  ...        2006  rejected      0    0    0      0         0   \n",
       "1            NaN  ...        2006  rejected      0    0    0      0         0   \n",
       "2            NaN  ...        2006  rejected      0    0    0      0         0   \n",
       "3            NaN  ...        2006  rejected      0    0    0      0         0   \n",
       "4            0.0  ...        2006  rejected      0    0    0      1         0   \n",
       "...          ...  ...         ...       ...    ...  ...  ...    ...       ...   \n",
       "1804869      NaN  ...      399385  approved      0    0    0      0         0   \n",
       "1804870      NaN  ...      399528  approved      0    0    0      0         0   \n",
       "1804871      NaN  ...      399457  approved      0    0    0      0         0   \n",
       "1804872      NaN  ...      399519  approved      0    0    0      0         0   \n",
       "1804873      NaN  ...      399318  approved      0    0    0      0         0   \n",
       "\n",
       "         sexual_explicit  identity_annotator_count  toxicity_annotator_count  \n",
       "0                    0.0                         0                         4  \n",
       "1                    0.0                         0                         4  \n",
       "2                    0.0                         0                         4  \n",
       "3                    0.0                         0                         4  \n",
       "4                    0.0                         4                        47  \n",
       "...                  ...                       ...                       ...  \n",
       "1804869              0.0                         0                         4  \n",
       "1804870              0.0                         0                         4  \n",
       "1804871              0.0                         0                         4  \n",
       "1804872              0.0                         0                        66  \n",
       "1804873              0.0                         0                         4  \n",
       "\n",
       "[1804874 rows x 45 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1804874, 45)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7097320</td>\n",
       "      <td>[ Integrity means that you pay your debts.]\\n\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7097321</td>\n",
       "      <td>This is malfeasance by the Administrator and t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7097322</td>\n",
       "      <td>@Rmiller101 - Spoken like a true elitist. But ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7097323</td>\n",
       "      <td>Paul: Thank you for your kind words.  I do, in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7097324</td>\n",
       "      <td>Sorry you missed high school. Eisenhower sent ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97315</th>\n",
       "      <td>7194635</td>\n",
       "      <td>He should lose his job for promoting mis-infor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97316</th>\n",
       "      <td>7194636</td>\n",
       "      <td>\"Thinning project is meant to lower fire dange...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97317</th>\n",
       "      <td>7194637</td>\n",
       "      <td>I hope you millennials are happy that you put ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97318</th>\n",
       "      <td>7194638</td>\n",
       "      <td>I'm thinking Kellyanne Conway (a.k.a. The Trum...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97319</th>\n",
       "      <td>7194639</td>\n",
       "      <td>I still can't figure why a pizza in AK cost mo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>97320 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id                                       comment_text\n",
       "0      7097320  [ Integrity means that you pay your debts.]\\n\\...\n",
       "1      7097321  This is malfeasance by the Administrator and t...\n",
       "2      7097322  @Rmiller101 - Spoken like a true elitist. But ...\n",
       "3      7097323  Paul: Thank you for your kind words.  I do, in...\n",
       "4      7097324  Sorry you missed high school. Eisenhower sent ...\n",
       "...        ...                                                ...\n",
       "97315  7194635  He should lose his job for promoting mis-infor...\n",
       "97316  7194636  \"Thinning project is meant to lower fire dange...\n",
       "97317  7194637  I hope you millennials are happy that you put ...\n",
       "97318  7194638  I'm thinking Kellyanne Conway (a.k.a. The Trum...\n",
       "97319  7194639  I still can't figure why a pizza in AK cost mo...\n",
       "\n",
       "[97320 rows x 2 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv(r'G:\\First_NLP\\jigsaw-unintended-bias-in-toxicity-classification\\test.csv',sep = None)\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(97320, 2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count    1.804874e+06\n",
      "mean     5.186124e+01\n",
      "std      4.682154e+01\n",
      "min      1.000000e+00\n",
      "25%      1.700000e+01\n",
      "50%      3.500000e+01\n",
      "75%      7.300000e+01\n",
      "max      8.410000e+02\n",
      "Name: text_len, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "train['text_len'] = train['comment_text'].apply(lambda x: len(x.split(' ')))\n",
    "print(train['text_len'].describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
